from llama_index.core import Document
from llama_index.core.node_parser import TokenTextSplitter

doc = Document(
    text=(
        "This is sentence 1. This is sentence 2. "
        "Sentence 3 here."
    ),
    metadata={"author": "Tianmin Zheng"}
)
# 文件分割器
splitter = TokenTextSplitter(
    chunk_size=12,
    chunk_overlap=1,
    separator=" "
)
nodes = splitter.get_nodes_from_documents([doc])
for node in nodes:
    print('===>')
    print(node.text)
    print(node.metadata)
